<?php 
error_reporting(1);
define('PATH', substr(dirname(__FILE__), 0, -13));
$cfg_config = include PATH . '/config/config.php';
include PATH . '/core/Mysql.class.php';

$size = 50;
$kid = isset($argv[1])?$argv[1]:0;
$db = new dbQuery($cfg_config['mysql']['hostname'], $cfg_config['mysql']['username'], $cfg_config['mysql']['password'], $cfg_config['mysql']['database']);
$db->connect($cfg_config['mysql']['hostname'], $cfg_config['mysql']['username'], $cfg_config['mysql']['password'], $cfg_config['mysql']['database']);
$mongo = new Mongo($cfg_config['mongo']['server']);
$mongodb = $mongo -> $cfg_config['mongo']['dbname'];
$collection = $mongodb -> $cfg_config['mongo']['tbname'];
while (true) {
	$sql = "select * from `video_keys` where  `kid`> $kid ORDER BY `kid` ASC  limit $size" ;
	$rr = $db -> query($sql);
	$uitems = array();
	if ($db -> affected_rows($rr) == 0) {
		die("\n\n Over ... \n\n");
    } while ($row = $db -> fetch_array($rr)) {
		$tkey = $row['key'];
		$kid = $row['kid'];
		$mid = md5($tkey);
        if ($rs = $collection -> findOne(array('_id' => $mid))) {
			echo "\r $kid  (" . count($rs['data']) . ") ignore...              ";
		} else {
			echo "\r $kid  (" . count($rs['data']) . ") start...              ";
			$uitems["https://openapi.youku.com/v2/searches/video/by_keyword.json?client_id=336d7dc0f7107e49&keyword=" . $tkey . "&count=60&public_type=all"] = $mid;
		} 
    }
    $start = $kid;
	$r = classic_curl(array_keys($uitems), 0);
	$c = 0;
    $uitems1 = array();
	foreach($r as $url => $iids) {
		if (count($iids) > 0) {
			$uitems1["https://openapi.youku.com/v2/videos/show_basic_batch.json?client_id=336d7dc0f7107e49&video_ids=" . implode(',', $iids)] = $uitems[$url];
		} else {
			echo "\r $kid   err...              ";
		} 
	} 
	$res = classic_curl(array_keys($uitems1), 1);
	foreach($res as $url => $items_tmp) {
		if (count($items_tmp) > 0) {
			$obj = array();
			$mid = $uitems1[$url];
			$obj['_id'] = $mid;
			$obj['data'] = $items_tmp ;
			$collection -> insert($obj);
			echo "\r   $mid  (" . count($items_tmp) . ") ok...              ";
		} 
	} 
} 
// 析出IIDS
function get_iids($txt) {
	$result = array();
	$dataarry = json_decode($txt, true);
	$dataarry['videos'] =isset($dataarry['videos'])?$dataarry['videos']:array();
	for ($i = 0;$i < count($dataarry['videos']);$i++) {
		$result[] = $dataarry['videos'][$i]['id'];
	} 
	return $result;
} 
// 析出视频数据
function get_items($txt) {
	$items_tmp = array();
	$dataarrys = json_decode($txt, true);
	$videos = isset($dataarrys['videos'])?$dataarrys['videos']:array();;
	foreach($videos as $v) {
		$desc = @str_replace('|', '', $v['description']);
		$title = str_replace('|', '', $v['title']);
		$items_tmp[] = $v['id'] . '|' . $title . '|' . preg_replace('%http://g[0-9]+.ykimg.com/%', '', $v['thumbnail']) . '|' . $v['duration'] . '|' . $v['category'] . '|' . $v['published'] . '|' . $desc;
	} 

	return $items_tmp;
} 
// 并发采集
function classic_curl($urls, $delay) {
	$queue = curl_multi_init();
	$map = array();
	foreach ($urls as $url) {
		// create cURL resources
		$ch = curl_init(); 
		// set URL and other appropriate options
		curl_setopt($ch, CURLOPT_URL, $url);

		curl_setopt($ch, CURLOPT_TIMEOUT, 4);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch, CURLOPT_HEADER, 0);
		curl_setopt($ch, CURLOPT_NOSIGNAL, true); 
		// add handle
		curl_multi_add_handle($queue, $ch);
		$map[$url] = $ch;
	} 
	$active = null; 
	// execute the handles
	do {
		$mrc = curl_multi_exec($queue, $active);
	} while ($mrc == CURLM_CALL_MULTI_PERFORM);

	while ($active > 0 && $mrc == CURLM_OK) {
		if (curl_multi_select($queue, 0.5) != -1) {
			do {
				$mrc = curl_multi_exec($queue, $active);
			} while ($mrc == CURLM_CALL_MULTI_PERFORM);
		} 
	} 
	$responses = array();
	foreach ($map as $url => $ch) {
		if ($delay == 0) {
			$responses[$url] = get_iids(curl_multi_getcontent($ch));
		} else {
			$responses[$url] = get_items(curl_multi_getcontent($ch));
		} 
		curl_multi_remove_handle($queue, $ch);
		curl_close($ch);
	} 
    curl_multi_close($queue);
	return $responses;
} 
